# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import json
import os

from itemadapter import ItemAdapter
from scrapy.utils.conf import closest_scrapy_cfg
proj_root = closest_scrapy_cfg()
assets_path=os.path.abspath(os.path.dirname('proj_root'))+'\\assets'
# 使用mysql数据库连接
import pymysql
# 用于存储下载下来的视频资料
class DouyinVideoObject(object):
    def __int__(self,author_sec_uid,anchor_name,anchor_icon_url,anchor_title,anchor_title_tag,author_pic,author_cover_url,author_signature,author_nickname,author_prevent_download,author_share_pic,can_forward,can_comment,author_id,can_share,can_show_comment,video_create_time,video_desc,video_group_id,video_share_link_desc,video_share_url,video_collect_count,video_comment_count,video_digg_count,video_play_count,video_share_count,video_allow_share,video_in_reviewing,video_is_delete,video_is_prohibited,text_extra,video_cover_url,video_duration,video_url_list,keyword):
        self.anchor_name=anchor_name
        self.anchor_icon_url=anchor_icon_url
        self.anchor_title=anchor_title
        self.anchor_title_tag=anchor_title_tag
        self.author_pic=author_pic
        self.author_cover_url=author_cover_url
        self.author_nickname=author_nickname
        self.author_sec_uid=author_sec_uid
        self.author_signature=author_signature
        self.author_prevent_download=author_prevent_download
        self.author_share_pic=author_share_pic
        self.can_forward=can_forward
        self.can_comment=can_comment
        self.author_id=author_id
        self.can_share=can_share
        self.can_show_comment=can_show_comment
        self.video_create_time=video_create_time
        self.video_desc=video_desc
        self.video_group_id=video_group_id
        self.video_share_link_desc=video_share_link_desc
        self.video_share_url=video_share_url
        self.video_collect_count=video_collect_count
        self.video_comment_count=video_comment_count
        self.video_digg_count=video_digg_count
        self.video_play_count=video_play_count
        self.video_share_count=video_share_count
        self.video_allow_share=video_allow_share
        self.video_in_reviewing=video_in_reviewing
        self.video_is_delete=video_is_delete
        self.video_is_prohibited=video_is_prohibited
        self.text_extra=text_extra
        self.video_cover_url=video_cover_url
        self.video_duration=video_duration
        self.video_url_list=video_url_list
        self.keyword=keyword


class Douyin1Pipeline:
    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def __init__(self, crawler):
        self.douyin1_withkeyword = json.load(open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1_withkeyword.json', 'r',encoding = 'utf-8'))['mysqlConnectParam']
        self.douyin1_userId = json.load(open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1_userId.json', 'r',encoding = 'utf-8'))['mysqlConnectParam']
        self.douyin1 = json.load(open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1.json', 'r',encoding = 'utf-8'))['mysqlConnectParam']
        self.crawler = crawler
    # 开启爬虫的时候初始化数据库
    def open_spider(self, spider):
        if spider.name == 'douyin1':
            self.conn_douyin1= pymysql.connect(host=self.douyin1['host'],port=self.douyin1['port'],user=self.douyin1['user'],password=self.douyin1['password'],database=self.douyin1['database'],charset=self.douyin1['charset'])
        elif spider.name == 'douyin1_withkeyword':
            self.conn_douyin1_withkeyword= pymysql.connect(host=self.douyin1_withkeyword['host'],port=self.douyin1_withkeyword['port'],user=self.douyin1_withkeyword['user'],password=self.douyin1_withkeyword['password'],database=self.douyin1_withkeyword['database'],charset=self.douyin1_withkeyword['charset'])
        else:
            self.conn_douyin1_userId = pymysql.connect(host=self.douyin1_userId['host'],
                                                            port=self.douyin1_userId['port'],
                                                            user=self.douyin1_userId['user'],
                                                            password=self.douyin1_userId['password'],
                                                            database=self.douyin1_userId['database'],
                                                            charset=self.douyin1_userId['charset'])
    def process_item(self, item, spider):
        if spider.name == 'douyin1' or spider.name=='douyin1_userId':
            spider.logger.warning('开始解析数据!')
            if spider.name=='douyin1':
                self.conn_douyin1.ping()
            elif spider.name=='douyin1_userId':
                self.conn_douyin1_userId.ping()
            item = dict(item)
            aweme_list=item['aweme_list']['aweme_list']
            for i in aweme_list:
                # 填充对象
                video_obj = DouyinVideoObject()
                video_obj.anchor_name='' if i.get('ancestor_info') is None else '' if i['ancestor_info'].get('uid') is None else i['ancestor_info']['uid']
                video_obj.anchor_icon_url='' if i.get('anchor_info') is None else '' if i['anchor_info'].get('icon') is None else json.dumps(i['anchor_info']['icon']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.anchor_title='' if i.get('anchor_info') is None else '' if i['anchor_info'].get('title') is None else i['anchor_info']['title']
                video_obj.anchor_title_tag='' if i.get('anchor_info') is None else '' if i['anchor_info'].get('title_tag') is None else i['anchor_info']['title_tag']
                video_obj.author_pic=json.dumps(i['author']['avatar_thumb']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.author_cover_url=json.dumps(i['author']['cover_url'][0]['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.author_prevent_download=i['author']['prevent_download']
                video_obj.author_share_pic=json.dumps(i['author']['share_info']['share_qrcode_url']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.author_nickname = i['author']['nickname']
                video_obj.author_sec_uid = i['author']['sec_uid']
                video_obj.author_signature = i['author'].get('signature') if i['author'].get('signature') else ''
                video_obj.author_id = i['author']['uid']
                video_obj.can_forward =i['aweme_control']['can_forward'] if spider.name=='douyin1' else i['author']['aweme_control']['can_forward']
                video_obj.can_comment = i['aweme_control']['can_comment'] if spider.name=='douyin1' else i['author']['aweme_control']['can_comment']
                video_obj.can_share = i['aweme_control']['can_share'] if spider.name=='douyin1' else i['author']['aweme_control']['can_share']
                video_obj.can_show_comment = i['aweme_control']['can_show_comment'] if spider.name=='douyin1' else i['author']['aweme_control']['can_show_comment']
                video_obj.video_create_time = i['create_time']
                video_obj.video_desc = i['desc']
                video_obj.video_group_id = i['group_id']
                video_obj.video_share_link_desc = i['share_info']['share_link_desc']
                video_obj.video_share_url = i['share_info']['share_url']
                video_obj.video_collect_count = i['statistics']['collect_count']
                video_obj.video_comment_count = i['statistics']['comment_count']
                video_obj.video_digg_count = i['statistics']['digg_count']
                video_obj.video_play_count = i['statistics']['play_count']
                video_obj.video_share_count = i['statistics']['share_count']
                video_obj.video_allow_share = i['status']['allow_share']
                video_obj.video_in_reviewing = i['status']['in_reviewing']
                video_obj.video_is_delete = i['status']['is_delete']
                video_obj.video_is_prohibited = i['status']['is_prohibited']
                video_obj.text_extra='/'.join([ci.get('hashtag_name') if ci.get('hashtag_name') is not None else '' for ci in i['text_extra']])
                video_obj.video_cover_url = json.dumps(i['video']['cover']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.video_duration = i['video']['duration']
                video_obj.video_url_list = json.dumps(i['video']['play_addr']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                # print(video_obj.__dict__)
                # 初始化数据库
                global cursor
                if spider.name == 'douyin1':
                    cursor=self.conn_douyin1.cursor()
                elif spider.name == 'douyin1_userId':
                    cursor=self.conn_douyin1_userId.cursor()
                # video_tuple=(i for i in video_obj.__dict__.values())
                try:
                    # 接下来就是去重的工作了
                    sql ='select text_extra,video_group_id,video_duration,video_share_url from video_info where text_extra=%s and video_group_id=%s and video_duration=%s and video_share_url=%s'
                    cursor.execute(sql,(video_obj.text_extra,video_obj.video_group_id,video_obj.video_duration,video_obj.video_share_url))
                    result=cursor.fetchall()
                    if not result:
                        sql='insert into video_info(anchor_name,anchor_icon_url,anchor_title,anchor_title_tag,author_pic,author_cover_url,author_nickname,author_sec_uid,author_prevent_download,author_share_pic,can_forward,can_comment,author_id,can_share,can_show_comment,video_create_time,video_desc,video_group_id,video_share_link_desc,video_share_url,video_collect_count,video_comment_count,video_digg_count,video_play_count,video_share_count,video_allow_share,video_in_reviewing,video_is_delete,video_is_prohibited,text_extra,video_cover_url,video_duration,video_url_list) ' \
                            'values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                        cursor.execute(sql,(video_obj.anchor_name,video_obj.anchor_icon_url,video_obj.anchor_title,video_obj.anchor_title_tag,video_obj.author_pic,video_obj.author_cover_url,video_obj.author_nickname,video_obj.author_sec_uid,video_obj.author_prevent_download,video_obj.author_share_pic,video_obj.can_forward,video_obj.can_comment,video_obj.author_id,video_obj.can_share,video_obj.can_show_comment,video_obj.video_create_time,video_obj.video_desc,video_obj.video_group_id,video_obj.video_share_link_desc,video_obj.video_share_url,video_obj.video_collect_count,video_obj.video_comment_count,video_obj.video_digg_count,video_obj.video_play_count,video_obj.video_share_count,video_obj.video_allow_share,video_obj.video_in_reviewing,video_obj.video_is_delete,video_obj.video_is_prohibited,video_obj.text_extra,video_obj.video_cover_url,video_obj.video_duration,video_obj.video_url_list))
                        result = cursor.fetchall()
                        spider.logger.warning('数据添加成功!')
                    else:
                        spider.logger.warning('数据已存在!!!!!!!!!!!!!!!!!!')
                except Exception as e:
                    spider.logger.error('解析数据发生错误:{}'.format(e))
                    if spider.name == 'douyin1':
                        self.conn_douyin1.rollback()
                    elif spider.name == 'douyin1_userId':
                        self.conn_douyin1_userId.rollback()
                finally:
                    cursor.close()
                    if spider.name == 'douyin1':
                        self.conn_douyin1.commit()
                    elif spider.name == 'douyin1_userId':
                        self.conn_douyin1_userId.commit()
            spider.logger.warning('数据解析完成!!!')
        elif spider.name == 'douyin1_withkeyword':
            spider.logger.warning('开始解析数据!')
            self.conn_douyin1_withkeyword.ping()
            item = dict(item)
            aweme_list=item['aweme_list']['data']
            for i in aweme_list:
                # 填充对象
                video_obj = DouyinVideoObject()
                video_obj.keyword=item['aweme_list']['global_doodle_config']['keyword']
                video_obj.anchor_name=i['aweme_mix_info']['mix_info']['mix_id'] if i.get('type') ==16 else i['aweme_info']['mix_info']['mix_id'] if i['aweme_info'].get('mix_info') is not None else ''
                video_obj.anchor_icon_url=json.dumps(i['aweme_mix_info']['mix_info']['cover_url']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i.get('type') ==16 else json.dumps(i['aweme_info']['mix_info']['cover_url']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i['aweme_info'].get('mix_info') is not None else ''
                video_obj.anchor_title=i['aweme_mix_info']['mix_info']['mix_name'] if i.get('type') ==16 else i['aweme_info']['mix_info']['mix_name'] if i['aweme_info'].get('mix_info') is not None else ''
                video_obj.anchor_title_tag=i['aweme_mix_info']['mix_info']['desc'] if i.get('type') ==16 else i['aweme_info']['mix_info']['desc'] if i['aweme_info'].get('mix_info') is not None else ''
                video_obj.author_pic=json.dumps(i['aweme_mix_info']['mix_info']['author']['avatar_thumb']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i.get('type') ==16 else json.dumps(i['aweme_info']['author']['avatar_thumb']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                video_obj.author_cover_url='' if i.get('type') ==16 else ''
                video_obj.author_prevent_download=(i['aweme_mix_info']['mix_items'][0].get('prevent_download') if i['aweme_mix_info']['mix_items'][0].get('prevent_download') is not None else '') if i.get('type') ==16 else (i['aweme_info']['prevent_download'] if i['aweme_info'].get('prevent_download') is not None else '')
                video_obj.author_share_pic=(json.dumps(i['aweme_mix_info']['mix_items'][0]['author']['share_info']['share_qrcode_url']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i['aweme_mix_info']['mix_items'][0]['author'].get('share_info') is not None else '')  if i.get('type') ==16 else (json.dumps(i['aweme_info']['author']['share_info']['share_qrcode_url']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i['aweme_info']['author'].get('share_info') is not None else "")
                video_obj.author_nickname = i['aweme_mix_info']['mix_info']['author']['nickname'] if i.get('type') ==16 else  i['aweme_info']['author']['nickname']
                video_obj.author_sec_uid = (i['aweme_mix_info']['mix_info']['author']['sec_uid'] if i['aweme_mix_info']['mix_info']['author'].get('sec_uid') else '') if i.get('type') ==16 else  (i['aweme_info']['author']['sec_uid'] if i['aweme_info']['author'].get('sec_uid') else '')
                video_obj.author_signature = (i['aweme_mix_info']['mix_info']['author'].get('signature') if i['aweme_mix_info']['mix_info']['author'].get('signature') is not None else'') if i.get('type') ==16 else (i['aweme_info']['author']['signature'] if i['aweme_info']['author'].get('signature') else '')
                video_obj.author_id = i['aweme_mix_info']['mix_info']['author']['uid'] if i.get('type') ==16 else i['aweme_info']['author']['uid']
                video_obj.can_forward = (i['aweme_mix_info']['mix_items'][0]['status']['self_see'] if i['aweme_mix_info']['mix_items'][0].get('status') is not None else '') if i.get('type') ==16 else (i['aweme_info']['status']['self_see'] if i['aweme_info'].get('status') is not None else '')
                video_obj.can_comment =(i['aweme_mix_info']['mix_items'][0]['status']['allow_comment'] if i['aweme_mix_info']['mix_items'][0].get('status') is not None else '') if i.get('type') ==16 else (i['aweme_info']['status']['allow_comment'] if i['aweme_info'].get('status') is not None else '')
                video_obj.can_share = (i['aweme_mix_info']['mix_items'][0]['status']['allow_share'] if i['aweme_mix_info']['mix_items'][0].get('status') is not None else '') if i.get('type') ==16 else (i['aweme_info']['status']['allow_share'] if i['aweme_info'].get('status') is not None else '')
                video_obj.can_show_comment = True if i.get('type') ==16 else True
                video_obj.video_create_time = (i['aweme_mix_info']['mix_items'][0]['create_time'] if i['aweme_mix_info']['mix_items'][0].get('create_time') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['create_time'] if i['aweme_info'].get('create_time') is not None else 0)
                video_obj.video_desc = (i['aweme_mix_info']['mix_items'][0]['desc'] if i['aweme_mix_info']['mix_items'][0].get('desc') is not None else '') if i.get('type') ==16 else (i['aweme_info']['desc'] if i['aweme_info'].get('desc') is not None else '')
                video_obj.video_group_id = i['aweme_mix_info']['mix_items'][0]['group_id'] if i.get('type') ==16 else i['aweme_info']['group_id']
                video_obj.video_share_link_desc = i['aweme_mix_info']['mix_items'][0]['desc'] if i.get('type') ==16 else (i['aweme_info']['share_info']['share_link_desc'] if i['aweme_info'].get('share_info') is not None else '')
                video_obj.video_share_url =('https://www.douyin.com/video/'+i['aweme_mix_info']['mix_items'][0]['group_id']) if i.get('type') ==16 else (i['aweme_info']['share_info']['share_url'] if i['aweme_info'].get('share_info') is not None else '')
                video_obj.video_collect_count = (i['aweme_mix_info']['mix_items'][0]['statistics']['collect_count'] if i['aweme_mix_info']['mix_items'][0].get('statistics') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['statistics']['collect_count'] if i['aweme_info'].get('statistics') is not None else 0)
                video_obj.video_comment_count = (i['aweme_mix_info']['mix_items'][0]['statistics']['comment_count'] if i['aweme_mix_info']['mix_items'][0].get('statistics') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['statistics']['comment_count'] if i['aweme_info'].get('statistics') is not None else 0)
                video_obj.video_digg_count = (i['aweme_mix_info']['mix_items'][0]['statistics']['digg_count'] if i['aweme_mix_info']['mix_items'][0].get('statistics') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['statistics']['digg_count'] if i['aweme_info'].get('statistics') is not None else 0)
                video_obj.video_play_count = (i['aweme_mix_info']['mix_items'][0]['statistics']['play_count'] if i['aweme_mix_info']['mix_items'][0].get('statistics') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['statistics']['play_count'] if i['aweme_info'].get('statistics') is not None else 0)
                video_obj.video_share_count =(i['aweme_mix_info']['mix_items'][0]['statistics']['share_count'] if i['aweme_mix_info']['mix_items'][0].get('statistics') is not None else 0) if i.get('type') ==16 else (i['aweme_info']['statistics']['share_count'] if i['aweme_info'].get('statistics') is not None else 0)
                video_obj.video_allow_share = (i['aweme_mix_info']['mix_items'][0]['status']['allow_share'] if i['aweme_mix_info']['mix_items'][0].get('status') is None else True) if i.get('type') ==16 else (i['aweme_info']['status']['allow_share'] if i['aweme_info'].get('status') is not None else True)
                video_obj.video_in_reviewing = (i['aweme_mix_info']['mix_items'][0]['status']['in_reviewing'] if i['aweme_mix_info']['mix_items'][0].get('status') is None else True) if i.get('type') ==16 else (i['aweme_info']['status']['in_reviewing'] if i['aweme_info'].get('status') is not None else True)
                video_obj.video_is_delete =(i['aweme_mix_info']['mix_items'][0]['status']['is_delete'] if i['aweme_mix_info']['mix_items'][0].get('status') is None else False) if i.get('type') ==16 else (i['aweme_info']['status']['is_delete'] if i['aweme_info'].get('status') is not None else False)
                video_obj.video_is_prohibited = (i['aweme_mix_info']['mix_items'][0]['status']['is_prohibited'] if i['aweme_mix_info']['mix_items'][0].get('status') is None else True) if i.get('type') ==16 else (i['aweme_info']['status']['is_prohibited'] if i['aweme_info'].get('status') is not None else True)
                if (i.get('type')==16 and i['aweme_mix_info']['mix_items'][0].get('text_extra') is not None)  or (i.get('type') !=16 and i['aweme_info'].get('text_extra') is not None):
                    video_obj.text_extra= '/'.join([c1['hashtag_name'] for c1 in i['aweme_mix_info']['mix_items'][0]['text_extra'] if c1.get('hashtag_name') is not None]) if i.get('type') ==16 else '/'.join([c1['hashtag_name'] for c1 in i['aweme_info']['text_extra'] if c1.get('hashtag_name') is not None])
                else:
                    video_obj.text_extra =''
                if (i.get('type') == 16 and i['aweme_mix_info']['mix_items'][0].get('video') is not None  and i['aweme_mix_info']['mix_items'][0].get('video').get('cover') is not None) or (i.get('type') != 16 and i['aweme_info'].get('video') is not None and  i['aweme_info']['video'].get('cover') is not None):
                    video_obj.video_cover_url = json.dumps(i['aweme_mix_info']['mix_items'][0]['video']['cover']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i.get('type') ==16 else json.dumps(i['aweme_info']['video']['cover']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                else:
                    video_obj.video_cover_url=''
                if (i.get('type') == 16 and i['aweme_mix_info']['mix_items'][0].get('video') is not None  and  i['aweme_mix_info']['mix_items'][0]['video'].get('play_addr') is not None) or (i.get('type') != 16 and i['aweme_info'].get('video') is not None and  i['aweme_info']['video'].get('play_addr') is not None):
                    video_obj.video_duration = i['aweme_mix_info']['mix_items'][0]['video']['play_addr']['data_size'] if i.get('type') ==16 else i['aweme_info']['video']['play_addr']['data_size']
                    video_obj.video_url_list = json.dumps(i['aweme_mix_info']['mix_items'][0]['video']['play_addr']['url_list']).replace(' ','').replace('\t','').replace('\n','') if i.get('type') ==16 else json.dumps(i['aweme_info']['video']['play_addr']['url_list']).replace(' ','').replace('\t','').replace('\n','')
                else:
                    video_obj.video_duration=0
                    video_obj.video_url_list=''
                # print(video_obj.__dict__)
                # 初始化数据库
                cursor=self.conn_douyin1_withkeyword.cursor()
                # video_tuple=(i for i in video_obj.__dict__.values())
                try:
                    # 接下来就是去重的工作了
                    sql ='select text_extra,video_group_id,video_duration from video_info where text_extra=%s and video_group_id=%s and video_duration=%s'
                    cursor.execute(sql,(video_obj.text_extra,video_obj.video_group_id,video_obj.video_duration))
                    result=cursor.fetchall()
                    if not result:
                        sql='insert into video_info(anchor_name,anchor_icon_url,anchor_title,anchor_title_tag,author_pic,author_cover_url,author_nickname,author_prevent_download,author_share_pic,can_forward,can_comment,author_id,can_share,can_show_comment,video_create_time,video_desc,video_group_id,video_share_link_desc,video_share_url,video_collect_count,video_comment_count,video_digg_count,video_play_count,video_share_count,video_allow_share,video_in_reviewing,video_is_delete,video_is_prohibited,text_extra,video_cover_url,video_duration,video_url_list,author_signature,keyword,author_sec_uid) ' \
                            'values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                        cursor.execute(sql,(video_obj.anchor_name,video_obj.anchor_icon_url,video_obj.anchor_title,video_obj.anchor_title_tag,video_obj.author_pic,video_obj.author_cover_url,video_obj.author_nickname,video_obj.author_prevent_download,video_obj.author_share_pic,video_obj.can_forward,video_obj.can_comment,video_obj.author_id,video_obj.can_share,video_obj.can_show_comment,video_obj.video_create_time,video_obj.video_desc,video_obj.video_group_id,video_obj.video_share_link_desc,video_obj.video_share_url,video_obj.video_collect_count,video_obj.video_comment_count,video_obj.video_digg_count,video_obj.video_play_count,video_obj.video_share_count,video_obj.video_allow_share,video_obj.video_in_reviewing,video_obj.video_is_delete,video_obj.video_is_prohibited,video_obj.text_extra,video_obj.video_cover_url,video_obj.video_duration,video_obj.video_url_list,video_obj.author_signature,video_obj.keyword,video_obj.author_sec_uid))
                        result = cursor.fetchall()
                        spider.logger.warning('数据添加成功!!!')
                    else:
                        spider.logger.warning('数据已存在!!!!!!!!!!!!!!!!!!')
                except Exception as e:
                    spider.logger.error(e)
                    self.conn_douyin1_withkeyword.rollback()
                finally:
                    cursor.close()
                    self.conn_douyin1_withkeyword.commit()
            spider.logger.warning('数据解析完成!!!')
        if item.get('has_more')==False:
            self.crawler.engine.close_spider(spider, "没有更多数据了!!!!")
            spider.logger.warning('数据结束爬取:没有更多数据了!!!!')
        return item
    def close_spider(self, spider):
        if spider.name == 'douyin1':
            self.conn_douyin1.close()
        if spider.name == 'douyin1_withkeyword':
            self.conn_douyin1_withkeyword.close()
